package com.gxljc.bear.crawler.itaogao.ynxxb;

import java.util.ArrayList;
import java.util.Date;
import java.util.List;

import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.log4j.Logger;

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.gxljc.commons.util.Pair;
import com.gxljc.commons.util.StringUtil;
import com.gxljc.bear.crawler.itaogao.NewsBaseCrawler;
import com.gxljc.bear.crawler.itaogao.NewsBaseTable;
import com.gxljc.bear.crawler.util.DateUtil;
import com.gxljc.bear.crawler.util.JsoupUtil;

/**
 * s
 * 
 * @author Administrator -> haitao
 * @since 2021年9月20日 下午6:31:50
 */
public class YnxxbDetailCrawler extends NewsBaseCrawler {

	/**
	 * 
	 */
	private static final long serialVersionUID = 8159239236057720033L;
	private static final Logger LOG = Logger.getLogger(YnxxbDetailCrawler.class);
	private static final String USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36";

	private static final String EDITION_URL_LIST = "http://www.ynxxb.com/Epaper/Elist/%s";
	private static final String PAGE_URL_LIST = "http://www.ynxxb.com%s";
	private static final String DEFAULT_DATE_FORMAT2 = "yyyy-M-d";

	public YnxxbDetailCrawler() {
		super("t_newspaper_ynxxb");
	}

	@Override
	protected List<String> genSeeds() {
		LOG.info(" genSeeds for ynxxb ");
		List<String> seeds = new ArrayList<>();
		try {

			for (String date : genDatesTillNow(DEFAULT_DATE_FORMAT2)) {
				// fetch url list by parser
				String request = String.format(EDITION_URL_LIST, date);
				String result = resultByHttpClient(request);
				JSONObject json = (JSONObject) JSONObject.parse(result);
				if (json == null) {
					System.exit(0);
				}
				JSONArray datas = (JSONArray) json.get("data");
				if (validateObject(datas)) {

					for (Object o : datas) {
						JSONObject json2 = (JSONObject) o;
						String areaList = (String) json2.get("ImgAreass");
						if (areaList != null) {
							List<String> urlList = extractAreaList(areaList);
							seeds.addAll(completeUrl(PAGE_URL_LIST, date, urlList));
						}
					}
				}

			}
		} catch (Exception e) {
			e.printStackTrace();
		}
		LOG.info("total seeds size="+seeds.size());
		return seeds;
	}

	@Override
	protected List<String> genNewSeeds() {
		List<String> seeds = new ArrayList<>();
		String date = DateUtil.dateFormat(new Date(), DEFAULT_DATE_FORMAT2);
		String request = String.format(EDITION_URL_LIST, date);
		LOG.info("--- genNewSeed of ynxxb ----");
		// today
		// String request = "http://www.ynxxb.com/Epaper/Elist/2016-9-20";
		String result = resultByHttpClient(request);
		JSONObject json = (JSONObject) JSONObject.parse(result);
		if (json == null) {
			System.exit(0);
		}
		JSONArray datas = (JSONArray) json.get("data");
		if (validateObject(datas)) {

			for (Object o : datas) {
				JSONObject json2 = (JSONObject) o;
				String areaList = (String) json2.get("ImgAreass");
				if (areaList != null) {
					List<String> urlList = extractAreaList(areaList);
					seeds.addAll(completeUrl(PAGE_URL_LIST, date, urlList));
				}
			}
		}
		LOG.info(String.format("-- genNewSeeds size=%s --", seeds.size()));
		return seeds;
	}

	@SuppressWarnings("unchecked")
	@Override
	protected Pair<NewsBaseTable, byte[]> genTable(String url) {
		NewsBaseTable table = new NewsBaseTable();
		String request = url;
		String result = "";
		Pair<String, byte[]> pair = fetResult(request, proxy);
		if (pair == null) {
			return new Pair<NewsBaseTable, byte[]>(table, "".getBytes());
		}
		result = pair.first;
		byte[] html = pair.second;
		if (result != null) {
			JSONObject jsonResult = (JSONObject) JSONObject.parse(result);
			if (jsonResult != null) {

				String title = (String) jsonResult.get("title");
				if (!StringUtil.isNullOrEmpty(title)) {
					table.setTitle(title);
				}
				String date = (String) jsonResult.get("publish_date");
				String[] boxes = date.split(" ");
				int leng = boxes.length;
				if (boxes != null && leng >= 3) {

					if (validateObject(boxes)) {
						// System.out.println("boxes="+boxes);
						table.setPublishDate(genDate(boxes[leng - 2] + " " + boxes[leng - 1], "yyyy-MM-dd HH:mm"));
					}
				}
				List<String> images = (List<String>) jsonResult.get("pic_list");
				LOG.info("images=" + images);
				if (validateObject(images)) {
					table = processImages(url, table, images);
				}
				List<String> paraList = (List<String>) jsonResult.get("para_list");
				if (paraList != null) {
					table.setContent(genContent(paraList, table.getImages()));
				}
				String subTitle = (String) jsonResult.get("sub_title");

				if (!StringUtil.isNullOrEmpty(subTitle)) {
					table.setSubTitle(subTitle);
				}
				
				//author -> source
				String author = (String) jsonResult.get("author");
				if (!StringUtil.isNullOrEmpty(author)) {
					table.setSource(author);
				}
				// set default value
				table.setId(StringUtil.genMD5Val(url));
				table.setCrawlUrl(url);
				table.setCrawlDate(new Date());
				table.setCrawlFlag(1);
			}
		}
		return new Pair<NewsBaseTable, byte[]>(table, html);
	}

	public String resultByHttpClient(String url) {
		try {
			return genGetMethod(url).getResponseBodyAsString();
		} catch (Exception e) {
			LOG.error("***fetch error by POST for url=" + url);
			e.printStackTrace();
		}
		return null;
	}

	public List<String> extractAreaList(String areaList) {
		return JsoupUtil.extractAttrList(areaList, "area", "href");
	}

	private PostMethod genGetMethod(String url) throws Exception {
		HttpClient httpClient = new HttpClient();
		PostMethod getMethod = new PostMethod(url);
		getMethod.setRequestHeader("User-Agent", USER_AGENT);
		getMethod.setRequestHeader("Cookie", "CNZZDATA3044191=cnzz_eid%3D2118504991-1474332739-%26ntime%3D1474332739");
		getMethod.setRequestHeader("Referer", "http://www.ynxxb.com/Epaper/2016-9-20");
		getMethod.setRequestHeader("Origin", "http://www.ynxxb.com");
		getMethod.setRequestHeader("Host", "www.ynxxb.com");
		getMethod.setRequestHeader("X-Requested-With", "XMLHttpRequest");
		getMethod.getParams().setContentCharset("UTF-8");
		httpClient.executeMethod(getMethod);
		httpClient.getHttpConnectionManager().getParams().setConnectionTimeout(5000);
		httpClient.getHttpConnectionManager().getParams().setSoTimeout(20000);
		return getMethod;
	}

	public static void main(String[] args) {
		// http://www.ynxxb.com/Content/2016-9/12/N211285627346
		// String url =
		// "http://paper.ce.cn/jjrb/html/2016-08/16/content_309133.htm";
		// url = "http://www.ynxxb.com/Content/2016-9/12/N211285627346";
		// YnxxbDetailCrawler crawler = new YnxxbDetailCrawler();
		// Pair<String, byte[]> result = crawler.fetResult(url, false);
		// if (result != null) {
		//
		// System.out.println(result.first);
		// }

		System.out.println(DateUtil.dateFormat(new Date(), DEFAULT_DATE_FORMAT));
		System.out.println(DateUtil.dateFormat(new Date(), DEFAULT_DATE_FORMAT2));
		String str = "云南信息报 2016-09-19 23:51";
		System.out.println(str.split(" ").length);

	}
}
